from scrapy.spider import BaseSpider
from scrapy.http import Request, Response
from scrapy.selector import HtmlXPathSelector

DEBUG = 1
DEBUG_DEPARTMENTS = [
        'Psychology',
        'Cognitive+Science',
        ]

DEPARTMENTS = [
        'Psychology',
        'Cognitive+Science',
        'Computer+Science',
        'Linguistics',
        'Philosophy',
        'History',
        'Media+Studies',
        'Information',
        'Anthropology',
        'Economics',
        'Education',
        'Sociology',
        'Molecular+and+Cell+Biology',
        'Neuroscience',
        'Integrative+Biology',
        'Music',
        'Vision+Science',
        'Native+American+Studies',
        'Political+Science',
        'Rhetoric',
        ]

SEMESTER = 'FL'

class CourseSpider(BaseSpider):
    name = "course-spider"
    allowed_domains = ["berkeley.edu"]
    start_urls = []

    def __init__(self, category=None):
        to_search = DEBUG_DEPARTMENTS if DEBUG else DEPARTMENTS
        for d in to_search:
            new_url = 'http://osoc.berkeley.edu/OSOC/osoc?p_term=%s&p_deptname=%s' % (SEMESTER, d)
            self.start_urls.append(new_url)

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        links = hxs.select('//a')

        for l in links:
            if l.select('font/b'):
                path = l.select('@href').extract()
                if path:
                    return Request('http://osoc.berkeley.edu'+path[0],
                            callback=self.parse)
